<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

<html lang="en">

<head>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  <title>LCOV - code analysis - src/caffe/layers/hdf5_data_layer.cpp</title>
  <link rel="stylesheet" type="text/css" href="../../../gcov.css">
</head>

<body>

  <table width="100%" border=0 cellspacing=0 cellpadding=0>
    <tr><td class="title">LCOV - code coverage report</td></tr>
    <tr><td class="ruler"><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>

    <tr>
      <td width="100%">
        <table cellpadding=1 border=0 width="100%">
          <tr>
            <td width="10%" class="headerItem">Current view:</td>
            <td width="35%" class="headerValue"><a href="../../../index.html">top level</a> - <a href="index.html">src/caffe/layers</a> - hdf5_data_layer.cpp<span style="font-size: 80%;"> (source / <a href="hdf5_data_layer.cpp.func-sort-c.html">functions</a>)</span></td>
            <td width="5%"></td>
            <td width="15%"></td>
            <td width="10%" class="headerCovTableHead">Hit</td>
            <td width="10%" class="headerCovTableHead">Total</td>
            <td width="15%" class="headerCovTableHead">Coverage</td>
          </tr>
          <tr>
            <td class="headerItem">Test:</td>
            <td class="headerValue">code analysis</td>
            <td></td>
            <td class="headerItem">Lines:</td>
            <td class="headerCovTableEntry">2</td>
            <td class="headerCovTableEntry">83</td>
            <td class="headerCovTableEntryLo">2.4 %</td>
          </tr>
          <tr>
            <td class="headerItem">Date:</td>
            <td class="headerValue">2020-09-11 22:50:33</td>
            <td></td>
            <td class="headerItem">Functions:</td>
            <td class="headerCovTableEntry">2</td>
            <td class="headerCovTableEntry">20</td>
            <td class="headerCovTableEntryLo">10.0 %</td>
          </tr>
          <tr>
            <td class="headerItem">Legend:</td>
            <td class="headerValueLeg">            Lines:
            <span class="coverLegendCov">hit</span>
            <span class="coverLegendNoCov">not hit</span>
</td>
            <td></td>
          </tr>
          <tr><td><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>
        </table>
      </td>
    </tr>

    <tr><td class="ruler"><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>
  </table>

  <table cellpadding=0 cellspacing=0 border=0>
    <tr>
      <td><br></td>
    </tr>
    <tr>
      <td>
<pre class="sourceHeading">          Line data    Source code</pre>
<pre class="source">
<a name="1"><span class="lineNum">       1 </span>            : #ifdef USE_HDF5</a>
<span class="lineNum">       2 </span>            : /*
<span class="lineNum">       3 </span>            : TODO:
<span class="lineNum">       4 </span>            : - load file in a separate thread (&quot;prefetch&quot;)
<span class="lineNum">       5 </span>            : - can be smarter about the memcpy call instead of doing it row-by-row
<span class="lineNum">       6 </span>            :   :: use util functions caffe_copy, and Blob-&gt;offset()
<span class="lineNum">       7 </span>            :   :: don't forget to update hdf5_daa_layer.cu accordingly
<span class="lineNum">       8 </span>            : - add ability to shuffle filenames if flag is set
<span class="lineNum">       9 </span>            : */
<span class="lineNum">      10 </span>            : #include &lt;fstream&gt;  // NOLINT(readability/streams)
<span class="lineNum">      11 </span>            : #include &lt;string&gt;
<span class="lineNum">      12 </span>            : #include &lt;vector&gt;
<span class="lineNum">      13 </span>            : 
<span class="lineNum">      14 </span>            : #include &quot;hdf5.h&quot;
<span class="lineNum">      15 </span>            : #include &quot;hdf5_hl.h&quot;
<span class="lineNum">      16 </span>            : #include &quot;stdint.h&quot;
<span class="lineNum">      17 </span>            : 
<span class="lineNum">      18 </span>            : #include &quot;caffe/layers/hdf5_data_layer.hpp&quot;
<span class="lineNum">      19 </span>            : #include &quot;caffe/util/hdf5.hpp&quot;
<span class="lineNum">      20 </span>            : 
<span class="lineNum">      21 </span>            : namespace caffe {
<a name="22"><span class="lineNum">      22 </span>            : </a>
<span class="lineNum">      23 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">      24 </span><span class="lineNoCov">          0 : HDF5DataLayer&lt;Dtype&gt;::~HDF5DataLayer&lt;Dtype&gt;() { }</span>
<span class="lineNum">      25 </span>            : 
<span class="lineNum">      26 </span>            : // Load data and label from HDF5 filename into the class property blobs.
<span class="lineNum">      27 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">      28 </span><span class="lineNoCov">          0 : void HDF5DataLayer&lt;Dtype&gt;::LoadHDF5FileData(const char* filename) {</span>
<span class="lineNum">      29 </span>            :   DLOG(INFO) &lt;&lt; &quot;Loading HDF5 file: &quot; &lt;&lt; filename;
<span class="lineNum">      30 </span><span class="lineNoCov">          0 :   hid_t file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);</span>
<span class="lineNum">      31 </span><span class="lineNoCov">          0 :   if (file_id &lt; 0) {</span>
<span class="lineNum">      32 </span><span class="lineNoCov">          0 :     LOG(FATAL) &lt;&lt; &quot;Failed opening HDF5 file: &quot; &lt;&lt; filename;</span>
<span class="lineNum">      33 </span>            :   }
<span class="lineNum">      34 </span>            : 
<span class="lineNum">      35 </span>            :   int top_size = this-&gt;layer_param_.top_size();
<span class="lineNum">      36 </span><span class="lineNoCov">          0 :   hdf_blobs_.resize(top_size);</span>
<span class="lineNum">      37 </span>            : 
<span class="lineNum">      38 </span>            :   const int MIN_DATA_DIM = 1;
<span class="lineNum">      39 </span>            :   const int MAX_DATA_DIM = INT_MAX;
<span class="lineNum">      40 </span>            : 
<span class="lineNum">      41 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; top_size; ++i) {</span>
<span class="lineNum">      42 </span><span class="lineNoCov">          0 :     hdf_blobs_[i] = shared_ptr&lt;Blob&lt;Dtype&gt; &gt;(new Blob&lt;Dtype&gt;());</span>
<span class="lineNum">      43 </span>            :     // Allow reshape here, as we are loading data not params
<span class="lineNum">      44 </span><span class="lineNoCov">          0 :     hdf5_load_nd_dataset(file_id, this-&gt;layer_param_.top(i).c_str(),</span>
<span class="lineNum">      45 </span>            :         MIN_DATA_DIM, MAX_DATA_DIM, hdf_blobs_[i].get(), true);
<span class="lineNum">      46 </span>            :   }
<span class="lineNum">      47 </span>            : 
<span class="lineNum">      48 </span><span class="lineNoCov">          0 :   herr_t status = H5Fclose(file_id);</span>
<span class="lineNum">      49 </span><span class="lineNoCov">          0 :   CHECK_GE(status, 0) &lt;&lt; &quot;Failed to close HDF5 file: &quot; &lt;&lt; filename;</span>
<span class="lineNum">      50 </span>            : 
<span class="lineNum">      51 </span>            :   // MinTopBlobs==1 guarantees at least one top blob
<span class="lineNum">      52 </span><span class="lineNoCov">          0 :   CHECK_GE(hdf_blobs_[0]-&gt;num_axes(), 1) &lt;&lt; &quot;Input must have at least 1 axis.&quot;;</span>
<span class="lineNum">      53 </span>            :   const int num = hdf_blobs_[0]-&gt;shape(0);
<span class="lineNum">      54 </span><span class="lineNoCov">          0 :   for (int i = 1; i &lt; top_size; ++i) {</span>
<span class="lineNum">      55 </span><span class="lineNoCov">          0 :     CHECK_EQ(hdf_blobs_[i]-&gt;shape(0), num);</span>
<span class="lineNum">      56 </span>            :   }
<span class="lineNum">      57 </span>            :   // Default to identity permutation.
<span class="lineNum">      58 </span>            :   data_permutation_.clear();
<span class="lineNum">      59 </span><span class="lineNoCov">          0 :   data_permutation_.resize(hdf_blobs_[0]-&gt;shape(0));</span>
<span class="lineNum">      60 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; hdf_blobs_[0]-&gt;shape(0); i++)</span>
<span class="lineNum">      61 </span><span class="lineNoCov">          0 :     data_permutation_[i] = i;</span>
<span class="lineNum">      62 </span>            : 
<span class="lineNum">      63 </span>            :   // Shuffle if needed.
<span class="lineNum">      64 </span><span class="lineNoCov">          0 :   if (this-&gt;layer_param_.hdf5_data_param().shuffle()) {</span>
<span class="lineNum">      65 </span><span class="lineNoCov">          0 :     std::random_shuffle(data_permutation_.begin(), data_permutation_.end());</span>
<span class="lineNum">      66 </span>            :     DLOG(INFO) &lt;&lt; &quot;Successfully loaded &quot; &lt;&lt; hdf_blobs_[0]-&gt;shape(0)
<span class="lineNum">      67 </span>            :                &lt;&lt; &quot; rows (shuffled)&quot;;
<span class="lineNum">      68 </span>            :   } else {
<span class="lineNum">      69 </span>            :     DLOG(INFO) &lt;&lt; &quot;Successfully loaded &quot; &lt;&lt; hdf_blobs_[0]-&gt;shape(0) &lt;&lt; &quot; rows&quot;;
<span class="lineNum">      70 </span>            :   }
<span class="lineNum">      71 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">      72 </span>            : 
<span class="lineNum">      73 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">      74 </span><span class="lineNoCov">          0 : void HDF5DataLayer&lt;Dtype&gt;::LayerSetUp(const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; bottom,</span>
<span class="lineNum">      75 </span>            :       const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; top) {
<span class="lineNum">      76 </span>            :   // Refuse transformation parameters since HDF5 is totally generic.
<span class="lineNum">      77 </span><span class="lineNoCov">          0 :   CHECK(!this-&gt;layer_param_.has_transform_param()) &lt;&lt;</span>
<span class="lineNum">      78 </span>            :       this-&gt;type() &lt;&lt; &quot; does not transform data.&quot;;
<span class="lineNum">      79 </span>            :   // Read the source to parse the filenames.
<span class="lineNum">      80 </span>            :   const string&amp; source = this-&gt;layer_param_.hdf5_data_param().source();
<span class="lineNum">      81 </span><span class="lineNoCov">          0 :   LOG(INFO) &lt;&lt; &quot;Loading list of HDF5 filenames from: &quot; &lt;&lt; source;</span>
<span class="lineNum">      82 </span><span class="lineNoCov">          0 :   hdf_filenames_.clear();</span>
<span class="lineNum">      83 </span><span class="lineNoCov">          0 :   std::ifstream source_file(source.c_str());</span>
<span class="lineNum">      84 </span><span class="lineNoCov">          0 :   if (source_file.is_open()) {</span>
<span class="lineNum">      85 </span>            :     std::string line;
<span class="lineNum">      86 </span><span class="lineNoCov">          0 :     while (source_file &gt;&gt; line) {</span>
<span class="lineNum">      87 </span><span class="lineNoCov">          0 :       hdf_filenames_.push_back(line);</span>
<span class="lineNum">      88 </span>            :     }
<span class="lineNum">      89 </span>            :   } else {
<span class="lineNum">      90 </span><span class="lineNoCov">          0 :     LOG(FATAL) &lt;&lt; &quot;Failed to open source file: &quot; &lt;&lt; source;</span>
<span class="lineNum">      91 </span>            :   }
<span class="lineNum">      92 </span><span class="lineNoCov">          0 :   source_file.close();</span>
<span class="lineNum">      93 </span><span class="lineNoCov">          0 :   num_files_ = hdf_filenames_.size();</span>
<span class="lineNum">      94 </span><span class="lineNoCov">          0 :   current_file_ = 0;</span>
<span class="lineNum">      95 </span><span class="lineNoCov">          0 :   LOG(INFO) &lt;&lt; &quot;Number of HDF5 files: &quot; &lt;&lt; num_files_;</span>
<span class="lineNum">      96 </span><span class="lineNoCov">          0 :   CHECK_GE(num_files_, 1) &lt;&lt; &quot;Must have at least 1 HDF5 filename listed in &quot;</span>
<span class="lineNum">      97 </span>            :     &lt;&lt; source;
<span class="lineNum">      98 </span>            : 
<span class="lineNum">      99 </span>            :   file_permutation_.clear();
<span class="lineNum">     100 </span><span class="lineNoCov">          0 :   file_permutation_.resize(num_files_);</span>
<span class="lineNum">     101 </span>            :   // Default to identity permutation.
<span class="lineNum">     102 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; num_files_; i++) {</span>
<span class="lineNum">     103 </span><span class="lineNoCov">          0 :     file_permutation_[i] = i;</span>
<span class="lineNum">     104 </span>            :   }
<span class="lineNum">     105 </span>            : 
<span class="lineNum">     106 </span>            :   // Shuffle if needed.
<span class="lineNum">     107 </span><span class="lineNoCov">          0 :   if (this-&gt;layer_param_.hdf5_data_param().shuffle()) {</span>
<span class="lineNum">     108 </span><span class="lineNoCov">          0 :     std::random_shuffle(file_permutation_.begin(), file_permutation_.end());</span>
<span class="lineNum">     109 </span>            :   }
<span class="lineNum">     110 </span>            : 
<span class="lineNum">     111 </span>            :   // Load the first HDF5 file and initialize the line counter.
<span class="lineNum">     112 </span><span class="lineNoCov">          0 :   LoadHDF5FileData(hdf_filenames_[file_permutation_[current_file_]].c_str());</span>
<span class="lineNum">     113 </span><span class="lineNoCov">          0 :   current_row_ = 0;</span>
<span class="lineNum">     114 </span>            : 
<span class="lineNum">     115 </span>            :   // Reshape blobs.
<span class="lineNum">     116 </span><span class="lineNoCov">          0 :   const int batch_size = this-&gt;layer_param_.hdf5_data_param().batch_size();</span>
<span class="lineNum">     117 </span>            :   const int top_size = this-&gt;layer_param_.top_size();
<span class="lineNum">     118 </span>            :   vector&lt;int&gt; top_shape;
<span class="lineNum">     119 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; top_size; ++i) {</span>
<span class="lineNum">     120 </span><span class="lineNoCov">          0 :     top_shape.resize(hdf_blobs_[i]-&gt;num_axes());</span>
<span class="lineNum">     121 </span><span class="lineNoCov">          0 :     top_shape[0] = batch_size;</span>
<span class="lineNum">     122 </span><span class="lineNoCov">          0 :     for (int j = 1; j &lt; top_shape.size(); ++j) {</span>
<span class="lineNum">     123 </span><span class="lineNoCov">          0 :       top_shape[j] = hdf_blobs_[i]-&gt;shape(j);</span>
<span class="lineNum">     124 </span>            :     }
<span class="lineNum">     125 </span><span class="lineNoCov">          0 :     top[i]-&gt;Reshape(top_shape);</span>
<span class="lineNum">     126 </span>            :   }
<span class="lineNum">     127 </span><span class="lineNoCov">          0 : }</span>
<a name="128"><span class="lineNum">     128 </span>            : </a>
<span class="lineNum">     129 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     130 </span><span class="lineNoCov">          0 : bool HDF5DataLayer&lt;Dtype&gt;::Skip() {</span>
<span class="lineNum">     131 </span>            :   int size = Caffe::solver_count();
<span class="lineNum">     132 </span>            :   int rank = Caffe::solver_rank();
<span class="lineNum">     133 </span><span class="lineNoCov">          0 :   bool keep = (offset_ % size) == rank ||</span>
<span class="lineNum">     134 </span>            :               // In test mode, only rank 0 runs, so avoid skipping
<span class="lineNum">     135 </span><span class="lineNoCov">          0 :               this-&gt;layer_param_.phase() == TEST;</span>
<span class="lineNum">     136 </span><span class="lineNoCov">          0 :   return !keep;</span>
<span class="lineNum">     137 </span>            : }
<span class="lineNum">     138 </span>            : 
<span class="lineNum">     139 </span>            : template&lt;typename Dtype&gt;
<span class="lineNum">     140 </span><span class="lineNoCov">          0 : void HDF5DataLayer&lt;Dtype&gt;::Next() {</span>
<span class="lineNum">     141 </span><span class="lineNoCov">          0 :   if (++current_row_ == hdf_blobs_[0]-&gt;shape(0)) {</span>
<span class="lineNum">     142 </span><span class="lineNoCov">          0 :     if (num_files_ &gt; 1) {</span>
<span class="lineNum">     143 </span><span class="lineNoCov">          0 :       ++current_file_;</span>
<span class="lineNum">     144 </span><span class="lineNoCov">          0 :       if (current_file_ == num_files_) {</span>
<span class="lineNum">     145 </span><span class="lineNoCov">          0 :         current_file_ = 0;</span>
<span class="lineNum">     146 </span><span class="lineNoCov">          0 :         if (this-&gt;layer_param_.hdf5_data_param().shuffle()) {</span>
<span class="lineNum">     147 </span><span class="lineNoCov">          0 :           std::random_shuffle(file_permutation_.begin(),</span>
<span class="lineNum">     148 </span>            :                               file_permutation_.end());
<span class="lineNum">     149 </span>            :         }
<span class="lineNum">     150 </span>            :         DLOG(INFO) &lt;&lt; &quot;Looping around to first file.&quot;;
<span class="lineNum">     151 </span>            :       }
<span class="lineNum">     152 </span><span class="lineNoCov">          0 :       LoadHDF5FileData(</span>
<span class="lineNum">     153 </span><span class="lineNoCov">          0 :         hdf_filenames_[file_permutation_[current_file_]].c_str());</span>
<span class="lineNum">     154 </span>            :     }
<span class="lineNum">     155 </span><span class="lineNoCov">          0 :     current_row_ = 0;</span>
<span class="lineNum">     156 </span><span class="lineNoCov">          0 :     if (this-&gt;layer_param_.hdf5_data_param().shuffle())</span>
<span class="lineNum">     157 </span><span class="lineNoCov">          0 :       std::random_shuffle(data_permutation_.begin(), data_permutation_.end());</span>
<span class="lineNum">     158 </span>            :   }
<span class="lineNum">     159 </span><span class="lineNoCov">          0 :   offset_++;</span>
<span class="lineNum">     160 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     161 </span>            : 
<span class="lineNum">     162 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     163 </span><span class="lineNoCov">          0 : void HDF5DataLayer&lt;Dtype&gt;::Forward_cpu(const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; bottom,</span>
<span class="lineNum">     164 </span>            :       const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; top) {
<span class="lineNum">     165 </span><span class="lineNoCov">          0 :   const int batch_size = this-&gt;layer_param_.hdf5_data_param().batch_size();</span>
<span class="lineNum">     166 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; batch_size; ++i) {</span>
<span class="lineNum">     167 </span><span class="lineNoCov">          0 :     while (Skip()) {</span>
<span class="lineNum">     168 </span><span class="lineNoCov">          0 :       Next();</span>
<span class="lineNum">     169 </span>            :     }
<span class="lineNum">     170 </span><span class="lineNoCov">          0 :     for (int j = 0; j &lt; this-&gt;layer_param_.top_size(); ++j) {</span>
<span class="lineNum">     171 </span><span class="lineNoCov">          0 :       int data_dim = top[j]-&gt;count() / top[j]-&gt;shape(0);</span>
<span class="lineNum">     172 </span><span class="lineNoCov">          0 :       caffe_copy(data_dim,</span>
<span class="lineNum">     173 </span><span class="lineNoCov">          0 :           &amp;hdf_blobs_[j]-&gt;cpu_data()[data_permutation_[current_row_]</span>
<span class="lineNum">     174 </span><span class="lineNoCov">          0 :             * data_dim], &amp;top[j]-&gt;mutable_cpu_data()[i * data_dim]);</span>
<span class="lineNum">     175 </span>            :     }
<span class="lineNum">     176 </span><span class="lineNoCov">          0 :     Next();</span>
<span class="lineNum">     177 </span>            :   }
<span class="lineNum">     178 </span><span class="lineNoCov">          0 : }</span>
<a name="179"><span class="lineNum">     179 </span>            : </a>
<span class="lineNum">     180 </span>            : #ifdef CPU_ONLY
<span class="lineNum">     181 </span><span class="lineNoCov">          0 : STUB_GPU_FORWARD(HDF5DataLayer, Forward);</span>
<span class="lineNum">     182 </span>            : #endif
<span class="lineNum">     183 </span>            : 
<span class="lineNum">     184 </span>            : INSTANTIATE_CLASS(HDF5DataLayer);
<a name="185"><span class="lineNum">     185 </span><span class="lineCov">          3 : REGISTER_LAYER_CLASS(HDF5Data);</span></a>
<span class="lineNum">     186 </span>            : 
<span class="lineNum">     187 </span><span class="lineCov">          3 : }  // namespace caffe</span>
<span class="lineNum">     188 </span>            : #endif  // USE_HDF5
</pre>
      </td>
    </tr>
  </table>
  <br>

  <table width="100%" border=0 cellspacing=0 cellpadding=0>
    <tr><td class="ruler"><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>
    <tr><td class="versionInfo">Generated by: <a href="http://ltp.sourceforge.net/coverage/lcov.php" target="_parent">LCOV version 1.12</a></td></tr>
  </table>
  <br>

</body>
</html>
